package com.g2room.driving.grap.article.kc;

import java.io.IOException;
import java.net.MalformedURLException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.alibaba.fastjson.JSON;
import com.g2room.driving.grap.article.to.WeixinMultiMsgInfo;
import com.g2room.driving.grap.article.to.WeixinTo;
import com.g2room.driving.model.po.ArticleGrap;
import com.g2room.driving.util.DbUtil;
import com.g2room.driving.util.HtmlUtil;
import com.g2room.driving.util.IpUtil;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;


public class GrapWeixinDataByProxy {
	static Connection connection = null;
	static PreparedStatement  ps = null;
	static Map<String, String> errorUrls = new HashMap<String, String>();
	/** 记录当前抓取的公众号url */
	static String currentUrl = "";
	
	private static final String baseUrl = "http://mp.weixin.qq.com";
	private static Map<String, String> urlMap = new HashMap<String, String>();
	// 界面UI
	// http://cq.qq.com/tour/
	// http://cq.qq.com/news/?qq=0&pt_src=3&ADUIN=252909344&ADSESSION=1461388995&ADTAG=CLIENT.QQ.5473_.0&ADPUBNO=26569
	static {
		urlMap.put("微车-玩车小师妹", "http://weixin.sogou.com/weixin?type=1&query=weiche_me&ie=utf8&_sug_=n&_sug_type_=");
		urlMap.put("开车小技巧", "http://weixin.sogou.com/weixin?type=1&query=%E5%BC%80%E8%BD%A6%E5%B0%8F%E6%8A%80%E5%B7%A7&ie=utf8&_sug_=n&_sug_type_=");
		urlMap.put("汽车知识", "http://weixin.sogou.com/weixin?type=1&query=xsjiaoliu&ie=utf8&_sug_=n&_sug_type_=");
		urlMap.put("汽车知识专家", "http://weixin.sogou.com/weixin?type=1&query=qczszj&ie=utf8&_sug_=n&_sug_type_=");
		urlMap.put("汽车保养维修知识", "http://weixin.sogou.com/weixin?type=1&query=che163&ie=utf8&_sug_=n&_sug_type_=");
		urlMap.put("重庆交巡警", "http://weixin.sogou.com/weixin?type=1&query=cqcq110&ie=utf8&_sug_=n&_sug_type_=");
	}
	
	private static Connection getConn() throws ClassNotFoundException, SQLException {
		if(null == connection) {
			connection = DbUtil.getConn();
		}
		return connection;
	}
	static Map<String, String> errMap = new HashMap<String, String>();
	public static void main(String[] args) throws Exception{
		grapWeixin();
    }
	
	public static void grapWeixin() throws Exception{
		GrapWeixinDataByProxy gkd = new GrapWeixinDataByProxy();
		gkd.getLink();
		// 最后输出失败的链接到数据库
		insertError2Db();
		DbUtil.closeConn();
    }
	
	public static String getJsSrc(String html) {
		String reg = "var msgList = '.*'";
		Pattern p=Pattern.compile(reg,Pattern.MULTILINE);
		Matcher m=p.matcher(html);
		while(m.find()){
			html = m.group(0);
		}
		// 处理html为标准的json
		return html.replace("var msgList = '{&quot;list&quot;:", "").replace("}'", "").replaceAll("&quot;", "\"").replaceAll("&amp;amp;", "&");
	}
	
	public static Document getDoc(String url) {
		String ip = IpUtil.getIp();
		Document doc = null;
		try {
			doc =Jsoup.connect(url)
//				  .data("query", "Java")   // 请求参数
					.header("X-Real-IP", ip)
					.header("x-forwarded-for", ip)
					.header("WL-Proxy-Client-IP", ip)
					.userAgent("Mozilla/5.0 (Windows NT 5.1; rv:6.0.2) Gecko/20100101 Firefox/6.0.2") // 设置 User-Agent 
//				.cookie("auth", "token") // 设置 cookie 
					.timeout(5000)           // 设置连接超时时间
					.get(); // get方式访问
		} catch (IOException e) {
			e.printStackTrace();
		}
		return doc;
	}
	
	
	/**
	 * 获取微信页面地址
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public void getLink() {
		for(String site : urlMap.keySet()) {
			System.out.println("抓取数据：" + site);
			Elements els = getDoc(urlMap.get(site)).getElementsByAttributeValue("class", "wx-rb bg-blue wx-rb_v1 _item");
			for(Element el : els) {
				getPage(el.attr("href"), el.getElementsByTag("h3").get(0).text());
				// 只取搜索出来的第一个
				break;
			}
		}
	}
	
	/**
	 * 获取分页数据
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public void getPage(String url, String categoryName) {
		System.out.println("start get page data!" + categoryName);
		String dataList = getJsSrc(getDoc(url).html());
		List<WeixinTo> weixins = JSON.parseArray(dataList, WeixinTo.class);
		List<ArticleGrap> details = new ArrayList<ArticleGrap>();
		// 遍历获取下面的子节点
		for(WeixinTo weixin : weixins) {
			ArticleGrap arti = new ArticleGrap();
			arti.setSource(categoryName);
			arti.setCategoryName(categoryName);
			arti.setTitle(filterEmoji(weixin.getApp_msg_ext_info().getTitle()));
			String publishTime = HtmlUtil.date2String(new Date(weixin.getComm_msg_info().getDatetime() * 1000), "yyyy-MM-dd");
			if(DbUtil.titleExist(arti.getTitle(), DbUtil.ConnType_KCJQ)) {
				break;
			} else {
				arti.setPublishTime(publishTime);
				String detailUrl = weixin.getApp_msg_ext_info().getContent_url().replaceAll("\\\\", "");
				arti.setSourceUrl(baseUrl + detailUrl);
				details.add(arti);
			}
			List<WeixinMultiMsgInfo> multiInfo = weixin.getApp_msg_ext_info().getMulti_app_msg_item_list();
			for(WeixinMultiMsgInfo muti : multiInfo) {
				ArticleGrap arti1 = new ArticleGrap();
				arti1.setSource(categoryName);
				arti1.setCategoryName(categoryName);
				arti1.setTitle(filterEmoji(muti.getTitle()));
				if(DbUtil.titleExist(arti1.getTitle(), DbUtil.ConnType_KCJQ)) {
					break;
				} else {
					arti1.setPublishTime(publishTime);
					String detailUrl = muti.getContent_url().replaceAll("\\\\", "");
					arti1.setSourceUrl(baseUrl + detailUrl);
					details.add(arti1);
				}
			}
		}
        
        for(ArticleGrap arti : details) {
        	try {
				Thread.sleep(IpUtil.genRandomNum() * 500);
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
        	getDetail(arti);
        }
	}
	
	/**
	 * 获取列表数据
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public void getDetail(ArticleGrap arti) {
		System.out.println("detail url ===" + arti.getSourceUrl());
		try {
			Document doc = getDoc(arti.getSourceUrl());
			doc.removeAttr("data-ratio").removeAttr("data-w");
			arti.setContent(handleQrCode(arti.getSource(), doc.getElementById("js_content")));
			insert2Db(arti);
		} catch (Exception e) {
			errMap.put(arti.getSourceUrl(), arti.getCategoryName());
			e.printStackTrace();
		}
	}
	
	public String handleQrCode(String source, Element content) {
		Elements es = content.children();
		int s = es.size() - 1;
		if(source.equals("微车-玩车小师妹")) {
			if(content.html().contains("http://mmbiz.qpic.cn/mmbiz/N1rLABFuReJMU5SfRqSI81d0lPHRnY47ZT95icdg3GovqwZxVVGkwVy3JViaLZXibIcSXtBuXic3rX2UYjDVjpCDdQ/0?wx_fmt=gif&wxfrom=5&wx_lazy=1")
					|| content.html().contains("http://mmbiz.qpic.cn/mmbiz/ke58Mm8ZOdNkvcydWylVCCaMdztkAO2uHEDlP8nFzPrcEodhHLVZl42b6fAaIhxIC3vrDDT0rp9AncLWjL1Dzw/640?wx_fmt=png&wxfrom=5&wx_lazy=1")
					|| content.html().contains("http://mmbiz.qpic.cn/mmbiz/N1rLABFuReJMU5SfRqSI81d0lPHRnY47ZT95icdg3GovqwZxVVGkwVy3JViaLZXibIcSXtBuXic3rX2UYjDVjpCDdQ/0?wx_fmt=gif&wxfrom=5&wx_lazy=1")) {
				// 带有关注二维码，则去掉最后三个标签
				for(int i=0; i<3; i++) {
					es.get(s--).remove();
				}
			}
			
		} else if(source.equals("开车小技巧")) {
			// 带有关注二维码，则去掉最后三个标签
			for(int i=0; i<3; i++) {
				es.get(s--).remove();
			}
			// 删除第一个图片
			es.get(0).remove();
		} else if(source.equals("汽车知识专家")) {
			// 带有关注二维码，则去掉最后三个标签
			for(int i=0; i<3; i++) {
				es.get(s--).remove();
			}
			// 删除第一个图片
			es.get(0).remove();
			es.get(1).remove();
		} else if(source.equals("汽车保养维修知识")) {
			if(s > 17) {
				// 带有关注二维码，则去掉最后三个标签
				for(int i=0; i<16; i++) {
					es.get(s--).remove();
				}
			}
			// 删除第一个图片
			es.get(0).remove();
		} else if(source.equals("重庆交巡警")) {
			Element e = null;
			if(es.size() < 3) {
				e = es.get(1);
				e.child(e.children().size() - 1).remove();
			} else {
				e = es.get(2);
				e.remove();
			}
			// 删除第一个图片
			es.get(0).remove();
		}
		String cont = "<div class=\"article\">" + content.html() + "</div>";
		cont = HtmlUtil.delAllTag(cont).replaceAll("img_width", "width").replaceAll("img_height", "height").replaceAll("data-src", "src");
		return filterEmoji(HtmlUtil.handleSpace(cont));
	}

	public static String filterEmoji(String source) {  
//        if(null != source && source.length() > 0){
//            return source.replaceAll("[\\ud800\\udc00-\\udbff\\udfff\\ud800-\\udfff]", "");  
//        }else{
//            return source;
//        }
//    
//	}
//	public static String filterEmoji(String source) {  
        if(source != null) {
            Pattern emoji = Pattern.compile ("[\ud83c\udc00-\ud83c\udfff]|[\ud83d\udc00-\ud83d\udfff]|[\u2600-\u27ff]",Pattern.UNICODE_CASE | Pattern . CASE_INSENSITIVE ) ;
            Matcher emojiMatcher = emoji.matcher(source);
            if ( emojiMatcher.find()) {
                source = emojiMatcher.replaceAll("");
                return source ; 
            }
        return source;
       }
       return source;  
    }
	
	int sqlCount = 0;
	public void insert2Db(ArticleGrap articles) {
		try {
			if(sqlCount == 0) ps = getConn().prepareStatement("insert into d_article_grap(title,content,type_name,category_name,source,source_url, state, publish_time) values(?,?,?,?,?,?,?,?)");
			ps.setString(1, articles.getTitle());
			ps.setString(2, articles.getContent());
			ps.setString(3, articles.getTypeName());
			ps.setString(4, articles.getCategoryName());
			ps.setString(5, articles.getSource());
			ps.setString(6, articles.getSourceUrl());
			ps.setInt(7, 8);
			ps.setString(8, articles.getPublishTime());
			ps.execute();
		} catch(Exception e) {
			e.printStackTrace();
		}
	}
	
	public static void insertError2Db() {
		try {
			PreparedStatement  ps = getConn().prepareStatement("insert into d_article_grap_err(title,source,source_url,type_name) values(?,?,?,?)");
			for(String url : errorUrls.keySet()) {
				ps.setString(1, errorUrls.get(url));
				ps.setString(2, "微信公众号");
				ps.setString(3, url);
				ps.setString(4, "导航和列表错误");
				ps.addBatch();
			}
			ps.executeBatch();
			if(null != ps) ps.close();
		} catch(Exception e) {
			e.printStackTrace();
		}
	}
}
